program SymantecAttackSignatures;

include <core.vhs>
include <netutil.vhs>

import core::*;
import core::Logging::*;
import core::Pattern::*;
import core::JobEngine::*;
import netutil::*;	

const ATTACK_SIGNATURE_REF = @'href="/business/security_response/attacksignatures/detail.jsp\?asid=(\d+)">(.+?)</a>';
const THREAT_REF = @'href="/security_response/writeup.jsp\?docid=(\d+-\d+-\d+-\d+)">(.+?)</a>';

const THREAT_URL_PATTERN = @'/security_response/writeup.jsp\?docid=(\d+-\d+-\d+-\d+)';

var numEntry = 0;
var failDownload = 0;
var localNumEntry;
var localFailDownload;

pattern<DataPart> DownloadIssueEntry($dataPattern=ATTACK_SIGNATURE_REF, $minOccur=0, $maxOccur = INFINITY, outfunc=null) {
	numEntry = numEntry + 1;
	var asid = this[1] + '';
	var name = this[2] + '';
	log("Signature %s: %s", asid, name);
	var url = 'http://www.symantec.com/security_response/attacksignatures/detail.jsp?asid=' + asid;
	var ifile = download(url, timeout=500000, cacheFile='webcache/as/as' + asid + '.html');
	if (ifile == null) {
		log("Unable to download %s", url);
		failDownload = failDownload + 1;
		failIssues.writeCsvLine([this[1], url]);
	}
	else {
		var ip = new SequenceStream(ifile, 4096);
		AttackSignature(input=ip, asid=asid, name=name, outfunc=outfunc);
		ip.close();
	}
}

pattern<DataPart> DownloadThreat($dataPattern=THREAT_REF, $minOccur=0, $maxOccur=INFINITY, outfunc=null) {
	numEntry = numEntry + 1;
	var docid = this[1] + '';
	var name = this[2] + '';
	log("Threat %s: %s", docid, name);
	var url = 'http://www.symantec.com/security_response/writeup.jsp?docid=' + docid;
	var ifile = download(url, timeout=500000, cacheFile='webcache/threat/threat' + docid + '.html');
	if (ifile == null) {
		log("Unable to download %s", url);
		failDownload = failDownload + 1;
		failIssues.writeCsvLine([this[1], url]);
	}
	else {
		var ip = new SequenceStream(ifile, 4096);
		Threat(input=ip, id=docid, name=name, outfunc=outfunc);
		ip.close();
	}
}

function downloadAllThreats(outfunc=null) {
	var urls = ['A', 'B','C','D','E','F','G','H','I','J','K','L','M','N','O','P','Q','R',
				'S','T','U','V','W','X','Y','Z','_1234567890'];
	var url = 'http://www.symantec.com/security_response/threatexplorer/azlisting.jsp';
	var file;
	var ip;
	for(var ch: urls) {
		file = download(url + (if (ch!='A') '?azid=' + ch else ''), cacheFile='webcache/threats-' + ch + '.html');
		if (file != null) {
			ip = new SequenceStream(file, 4096);
			DownloadThreat(input=ip, outfunc=outfunc);
			ip.close();
		}
	}
}

pattern AttackSignature($asid, $name, $maxOccur=1, $minOccur=0, outfunc) {
	// find for link to threats and/or other attacks
	
	var url;
	var tid;
	var haveLink = false;
	var haveCve = false;
	var matcher = match(this.LocalInput, 'href="(http://securityresponse.symantec.com/avcenter/venc/data/([^"]*))"');
	
	while (matcher.find()) {
		haveLink = true;
		url = matcher.group(1);
		tid = resolveThreatID(url);
		log("\t Link to threat: %s --> %s", matcher.group(2), tid);
		outfunc?(struct { output='LINK', aid=asid, link='threat', value=tid});
	}
	
	matcher = match(this.LocalInput, THREAT_URL_PATTERN);
	while (matcher.find()) {
		haveLink = true;
		tid = matcher.group(1);
		log("\t Link to threat [docid]: %s", tid);
		outfunc?(struct { output='LINK', aid=asid, link='threat', value=tid});
	}
	
	if (!haveLink)
		outfunc?(struct { output='LINK', aid=asid, link='none', value=''} );
	
	// function	
	var searchCve = function(input, type) {
		var cvematch = match(input, @'(CVE|CAN)-(\d+-\d+)');
		var foundCVE = map [];
		var cve;
		while (cvematch.find()) {
			haveCve = true;
			cve = cvematch.group(2);
			if (!foundCVE.containsKey(cve)) {
				log("\t CVE found (%s): %s", type, cve);
				foundCVE.put(cve, true);
				outfunc?(struct { output='CVE', aid=asid, type=type, cve=cve, name=name });
			}
		}
	}
	
	// find CVE not in Additional Reference Section
	var findCVESomeWhere = pattern($minOccur=0) {		
		locate_end('Additional References');
		//println('\tLook for CVE in the page...');
		searchCve(this.LocalInput, 'PAGE');
	}
	
	var findCVEinReference = pattern($minOccur=0) {
		//println('\tLook for CVE in the Reference...');
		searchCve(this.LocalInput, 'REF');
	}
	
	findCVESomeWhere();
	findCVEinReference();
	
	if (!haveCve)
		outfunc?(struct { output='CVE', aid=asid, type='none', cve='', name=name });
}

pattern Threat($id, $name, $maxOccur=1, $minOccur=0, outfunc) {
	var url;
	var vid;
	var haveLink = false;
	var haveCve = false;
	var matcher = match(this.LocalInput, 'href="(http://securityresponse.symantec.com/avcenter/venc/data/([^"]*))"');
	
	while (matcher.find()) {
		haveLink = true;
		url = matcher.group(1);
		vid = resolveThreatID(url);
		log("\t Link to threat: %s --> %s", matcher.group(2), vid);
		outfunc?(struct { output='LINK', tid=id, link='threat', value=vid});
	}
	
	matcher = match(this.LocalInput, THREAT_URL_PATTERN);
	while (matcher.find()) {
		vid = matcher.group(1);
		if (vid != id) {
			haveLink = true;		
			log("\t Link to threat [docid]: %s", vid);
			outfunc?(struct { output='LINK', tid=id, link='threat', value=vid});
		}
	}
	
	matcher = match(this.LocalInput, @'security_response/attacksignatures/detail.jsp?asid=(\d+)');
	while (matcher.find()) {
		vid = matcher.group(1);
		if (vid != id) {
			haveLink = true;		
			log("\t Link to signature : %s", vid);
			outfunc?(struct { output='LINK', tid=id, link='attack', value=vid});
		}
	}
	
	if (!haveLink)
		outfunc?(struct { output='LINK', tid=id, link='none', value=''} );
	
	// function	
	var searchCve = function(input, type) {
		var cvematch = match(input, @'(CVE|CAN)-(\d+-\d+)');
		var foundCVE = map [];
		var cve;
		while (cvematch.find()) {
			haveCve = true;
			cve = cvematch.group(2);
			if (!foundCVE.containsKey(cve)) {
				log("\t CVE found (%s): %s", type, cve);
				foundCVE.put(cve, true);
				outfunc?(struct { output='CVE', tid=id, type=type, cve=cve, name=name });
			}
		}
	}
	
	// find CVE not in Additional Reference Section
	var findCVESomeWhere = pattern($minOccur=0) {		
		//locate_end('Additional References');
		//println('\tLook for CVE in the page...');
		searchCve(this.LocalInput, 'PAGE');
	}
	
	var findCVEinReference = pattern($minOccur=0) {
		//println('\tLook for CVE in the Reference...');
		locate('CVE References');
		locate_end('</dd>');
		searchCve(this.LocalInput, 'REF');
	}
	findCVEinReference();
	findCVESomeWhere();

	if (!haveCve)
		outfunc?(struct { output='CVE', tid=id, type='none', cve='', name=name });
}

function resolveThreatID(url) {
	var f = download(url);
	if (f != null) {
		var ip = new SequenceStream(f, 4096);
		var matcher = match(ip, THREAT_URL_PATTERN);
		var result = if (matcher.find()) matcher.group(1) else null;
		ip.close();
		return result;
	}
	else
		return null;
}



function main() {
	Logging::configureLog("process-files.log");
	netutil::setExpiredSecPeriod(SEC_PER_MONTH); 
	log("Starting...");
	//var filename = download('http://securityresponse.symantec.com/avcenter/venc/data/adware.180search.html');
	
	var asCve = new CsvFile('as-cve.csv');
	var asLink = new CsvFile('as-link.csv');
	//var thCve = new CsvFile('th-cve.csv');
	//var thLink = new CsvFile('th-link.csv');
	
	asCve.writeln("aid,type,cve,name");
	asLink.writeln("aid,link,value");
	//thCve.writeln("tid,type,cve,name");
	//thLink.writeln("tid,link,value");
	
	var asOutput = function(data) {
		if (data.output == 'LINK') {
			asLink.writeCsvLine([data.aid, data.link, data.value]);
		}
		else
			asCve.writeCsvLine([data.aid, data.type, data.cve, data.name]);
	}
	
	var thOutput = function(data) {
		if (data.output == 'LINK') {
			thLink.writeCsvLine([data.tid, data.link, data.value]);
		}
		else
			thCve.writeCsvLine([data.tid, data.type, data.cve, data.name]);
	}
		
	//downloadAllThreats(outfunc=thOutput);
	
	var file = download('http://www.symantec.com/security_response/attacksignatures/', 
		timeout=500000, cacheFile='webcache/attacksignatures.html');
	if (file != null)
		DownloadIssueEntry(input = new SequenceStream(file, 4096), outfunc=asOutput);
	log("Completed");
	log("%d entries processed, %d entries fail.", numEntry, failDownload);
}
